import os

import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
from collections import Counter

import summarizer_data_utils
import summarizer_model_utils
import Summarizer

WARNING: Logging before flag parsing goes to stderr.
W0312 00:13:22.113631 140590899050240 __init__.py:56] Some hub symbols are not available because TensorFlow version is less than 1.14

import gc
gc.collect()

0

raw_texts = [line.strip() for line in open('./data/train.article.txt', 'r')]
raw_summaries = [line.strip() for line in open('./data/train.title.txt', 'r')]

print(len(raw_texts),len(raw_summaries))

3803957 3803957

import tensorflow as tf
with tf.device('/gpu:0'):
    a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')
    b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')
    c = tf.matmul(a, b)

with tf.Session() as sess:
    print (sess.run(c))

[[22. 28.]
 [49. 64.]]

import nltk
import pickle
nltk.download('punkt')


processed_texts, processed_summaries, words_counted = summarizer_data_utils.preprocess_texts_and_summaries(
    raw_texts,
    raw_summaries,
    keep_most=False
)



# with open('./data/processed_texts.pkl', 'wb') as f:
#     pickle.dump(processed_texts, f)

# with open('./data/processed_summaries.pkl', 'wb') as f:
#     pickle.dump(processed_summaries, f)

    
# with open('./data/words_counted.pkl', 'wb') as f:
#     pickle.dump(words_counted, f)


specials = ["<EOS>", "<SOS>","<PAD>","<UNK>"]
word2ind, ind2word,  missing_words = summarizer_data_utils.create_word_inds_dicts(words_counted,
                                                                       specials = specials)
print(len(word2ind), len(ind2word), len(missing_words))

[nltk_data] Downloading package punkt to /home/jupyter/nltk_data...
[nltk_data]   Package punkt is already up-to-date!

Processing Time:  1603.229953289032
100900 100900 0

# In[10]:

print("Embedding starts")
embed = hub.Module("https://tfhub.dev/google/Wiki-words-250/1")
emb = embed([key for key in word2ind.keys()])

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    sess.run(tf.tables_initializer())
    embedding = sess.run(emb)


# In[11]:


print(embedding.shape)

Embedding starts
INFO:tensorflow:Saver not created because there are no variables in the graph to restore

I0311 04:46:32.952139 140331509065472 saver.py:1483] Saver not created because there are no variables in the graph to restore

(100900, 250)

np.save('tf_hub_embedding_opinion.npy', embedding)
print("Embedding done")

Embedding done

#exporting processed texts
converted_texts, unknown_words_in_texts = summarizer_data_utils.convert_to_inds(processed_texts,
                                                                                word2ind,
                                                                                eos = False)
converted_summaries, unknown_words_in_summaries = summarizer_data_utils.convert_to_inds(processed_summaries,
                                                                                        word2ind,
                                                                                        eos = True,
                                                                                        sos = True)

# model hyperparametes
num_layers_encoder = 10
num_layers_decoder = 10
rnn_size_encoder = 256
rnn_size_decoder = 256

batch_size = 64
epochs = 5
clip = 5
keep_probability = 0.5
learning_rate = 0.0005
max_lr=0.005
learning_rate_decay_steps = 700
learning_rate_decay = 0.90


pretrained_embeddings_path = 'tf_hub_embedding_opinion.npy'
summary_dir = os.path.join('./tensorboard', str('Nn_' + str(rnn_size_encoder) + '_Lr_' + str(learning_rate)))


use_cyclic_lr = True
inference_targets=True


# In[15]:


print(len(converted_summaries))


# In[16]:


len_to_use = round(len(converted_summaries)*0.9)


# In[17]:


# build graph and train the model 
summarizer_model_utils.reset_graph()
summarizer = Summarizer.Summarizer(word2ind,
                                   ind2word,
                                   save_path='./models/GIGA/my_model',
                                   mode='TRAIN',
                                   num_layers_encoder = num_layers_encoder,
                                   num_layers_decoder = num_layers_decoder,
                                   rnn_size_encoder = rnn_size_encoder,
                                   rnn_size_decoder = rnn_size_decoder,
                                   batch_size = batch_size,
                                   clip = clip,
                                   keep_probability = keep_probability,
                                   learning_rate = learning_rate,
                                   max_lr=max_lr,
                                   learning_rate_decay_steps = learning_rate_decay_steps,
                                   learning_rate_decay = learning_rate_decay,
                                   epochs = epochs,
                                   pretrained_embeddings_path = pretrained_embeddings_path,
                                   use_cyclic_lr = use_cyclic_lr)
                                  # summary_dir = summary_dir) 
# hidden training output.
# both train and validation loss decrease nicely.


# In[ ]:


summarizer.build_graph()
summarizer.train(converted_texts[:len_to_use], 
                 converted_summaries[:len_to_use],
                 validation_inputs=converted_texts[len_to_use:],
                 validation_targets=converted_summaries[len_to_use:])

3803957
WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.

W0312 00:55:02.732816 140590899050240 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.

Loaded pretrained embeddings.
WARNING:tensorflow:From /home/jupyter/Summarizer.py:157: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.

W0312 00:55:04.059805 140590899050240 deprecation.py:323] From /home/jupyter/Summarizer.py:157: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.

WARNING:tensorflow:From /home/jupyter/Summarizer.py:254: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API

W0312 00:55:04.067770 140590899050240 deprecation.py:323] From /home/jupyter/Summarizer.py:254: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API

WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API

W0312 00:55:04.069038 140590899050240 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API

WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.

W0312 00:55:04.201420 140590899050240 deprecation.py:323] From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.

summarizer_model_utils.reset_graph()
summarizer = Summarizer.Summarizer(word2ind,
                                   ind2word,
                                   './models/GIGA/my_model',
                                   'INFER',
                                   num_layers_encoder = num_layers_encoder,
                                   num_layers_decoder = num_layers_decoder,
                                   batch_size = len(converted_texts[:50]),
                                   clip = clip,
                                   keep_probability = 1.0,
                                   learning_rate = 0.0,
                                   beam_width = 5,
                                   rnn_size_encoder = rnn_size_encoder,
                                   rnn_size_decoder = rnn_size_decoder,
                                   inference_targets = True,
                                   pretrained_embeddings_path = pretrained_embeddings_path)

summarizer.build_graph()
preds = summarizer.infer(converted_texts[:50],
                         restore_path =  './models/GIGA/my_model',
                         targets = converted_summaries[:50])

# show results
summarizer_model_utils.sample_results(preds,
                                      ind2word,
                                      word2ind,
                                      converted_summaries[:50],
                                      converted_texts[:50])

Loaded pretrained embeddings.
Graph built.